namespace CE.Parsers.Java
{
	using System;
	using System.Collections.Generic;
	using System.IO;

	//-----------------------------------------------------------------------------------
	// Buffer
	//-----------------------------------------------------------------------------------
	public class Buffer
	{
		#region Fields

		// This Buffer supports the following cases:
		// 1) seekable stream (file)
		//    a) whole stream in buffer
		//    b) part of stream in buffer
		// 2) non seekable stream (network, console)
		public const int EOF = char.MaxValue + 1;

		const int MAX_BUFFER_LENGTH = MIN_BUFFER_LENGTH * 64; // 64KB
		const int MIN_BUFFER_LENGTH = 1024; // 1KB

		byte[] buf; // input buffer
		int bufLen; // length of buffer
		int bufPos; // current position in buffer
		int bufStart; // position of first byte in buffer relative to input stream
		int fileLen; // length of input stream (may change if the stream is no file)
		bool isUserStream; // was the stream opened by the user?
		Stream stream; // input stream (seekable)

		#endregion Fields

		#region Constructors

		public Buffer(Stream s, bool isUserStream)
		{
			stream = s; this.isUserStream = isUserStream;

			if (stream.CanSeek) {
				fileLen = (int)stream.Length;
				bufLen = Math.Min(fileLen, MAX_BUFFER_LENGTH);
				bufStart = Int32.MaxValue; // nothing in the buffer so far
			} else {
				fileLen = bufLen = bufStart = 0;
			}

			buf = new byte[(bufLen > 0) ? bufLen : MIN_BUFFER_LENGTH];
			if (fileLen > 0) Pos = 0; // setup buffer to position 0 (start)
			else bufPos = 0; // index 0 is already after the file, thus Pos = 0 is invalid
			if (bufLen == fileLen && stream.CanSeek) Close();
		}

		protected Buffer(Buffer b)
		{
			// called in UTF8Buffer constructor
			buf = b.buf;
			bufStart = b.bufStart;
			bufLen = b.bufLen;
			fileLen = b.fileLen;
			bufPos = b.bufPos;
			stream = b.stream;
			// keep destructor from closing the stream
			b.stream = null;
			isUserStream = b.isUserStream;
		}

		~Buffer()
		{
			Close();
		}

		#endregion Constructors

		#region Properties

		public int Pos
		{
			get { return bufPos + bufStart; }
			set{
				if (value >= fileLen && stream != null && !stream.CanSeek) {
					// Wanted position is after buffer and the stream
					// is not seek-able e.g. network or console,
					// thus we have to read the stream manually till
					// the wanted position is in sight.
					while (value >= fileLen && ReadNextStreamChunk() > 0) ;
				}

				if (value < 0 || value > fileLen) {
					throw new FatalError("buffer out of bounds access, position: " + value);
				}

				if (value >= bufStart && value < bufStart + bufLen) { // already in buffer
					bufPos = value - bufStart;
				} else if (stream != null)   { // must be swapped in
					stream.Seek(value, SeekOrigin.Begin);
					bufLen = stream.Read(buf, 0, buf.Length);
					bufStart = value; bufPos = 0;
				} else {
					// set the position to the end of the file, Pos will return fileLen.
					bufPos = fileLen - bufStart;
				}
			}
		}

		#endregion Properties

		#region Methods

		public string GetString(int beg, int end)
		{
			int len = end - beg;
			char[] buf = new char[len];
			int oldPos = Pos;
			Pos = beg;
			for (int i = 0; i < len; i++) buf[i] = (char)Read();
			Pos = oldPos;
			return new String(buf);
		}

		public int Peek()
		{
			int curPos = Pos;
			int ch = Read();
			Pos = curPos;
			return ch;
		}

		public virtual int Read()
		{
			if (bufPos < bufLen) {
				return buf[bufPos++];
			} else if (Pos < fileLen)   {
				Pos = Pos; // shift buffer start to Pos
				return buf[bufPos++];
			} else if (stream != null && !stream.CanSeek && ReadNextStreamChunk() > 0)   {
				return buf[bufPos++];
			} else {
				return EOF;
			}
		}

		protected void Close()
		{
			if (!isUserStream && stream != null) {
				stream.Close();
				stream = null;
			}
		}

		// Read the next chunk of bytes from the stream, increases the buffer
		// if needed and updates the fields fileLen and bufLen.
		// Returns the number of bytes read.
		private int ReadNextStreamChunk()
		{
			int free = buf.Length - bufLen;
			if (free == 0) {
				// in the case of a growing input stream
				// we can neither seek in the stream, nor can we
				// foresee the maximum length, thus we must adapt
				// the buffer size on demand.
				byte[] newBuf = new byte[bufLen * 2];
				Array.Copy(buf, newBuf, bufLen);
				buf = newBuf;
				free = bufLen;
			}
			int read = stream.Read(buf, bufLen, free);
			if (read > 0) {
				fileLen = bufLen = (bufLen + read);
				return read;
			}
			// end of stream reached
			return 0;
		}

		#endregion Methods
	}

	//-----------------------------------------------------------------------------------
	// Scanner
	//-----------------------------------------------------------------------------------
	public class Scanner
	{
		#region Fields

		public Buffer buffer; // scanner buffer

		const char EOL = '\n';
		const int eofSym = 0; /* pdt */
		const int maxT = 101;
		const int noSym = 101;

		int ch; // current input character
		int col; // column number of current character
		int line; // line number of current character
		int oldEols; // EOLs that appeared in a comment;
		int pos; // byte position of current character
		Token pt; // current peek token
		Dictionary<int, int> start; // maps first token character to start state
		Token t; // current token
		int tlen; // length of current token
		Token tokens; // list of tokens already peeked (first token is a dummy)
		char[] tval = new char[128]; // text of current token

		#endregion Fields

		#region Constructors

		public Scanner(string fileName)
		{
			try {
				Stream stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read);
				buffer = new Buffer(stream, false);
				Init();
			}
			catch (IOException) {
				throw new FatalError("Cannot open file " + fileName);
			}
		}

		public Scanner(Stream s)
		{
			buffer = new Buffer(s, true);
			Init();
		}

		#endregion Constructors

		#region Methods

		// peek for the next token, ignore pragmas
		public Token Peek()
		{
			if (pt.next == null) {
				do {
					pt = pt.next = NextToken();
				} while (pt.kind > maxT); // skip pragmas
			} else {
				do {
					pt = pt.next;
				} while (pt.kind > maxT);
			}
			return pt;
		}

		// make sure that peeking starts at the current scan position
		public void ResetPeek()
		{
			pt = tokens;
		}

		// get the next token (possibly a token already seen during peeking)
		public Token Scan()
		{
			if (tokens.next == null) {
				return NextToken();
			} else {
				pt = tokens = tokens.next;
				return tokens;
			}
		}

		void AddCh()
		{
			if (tlen >= tval.Length) {
				char[] newBuf = new char[2 * tval.Length];
				Array.Copy(tval, 0, newBuf, 0, tval.Length);
				tval = newBuf;
			}
			tval[tlen++] = (char)ch;
			NextCh();
		}

		void CheckLiteral()
		{
			switch (t.val) {
			    case "boolean": t.kind = 6; break;
			    case "byte": t.kind = 7; break;
			    case "char": t.kind = 8; break;
			    case "class": t.kind = 9; break;
			    case "double": t.kind = 10; break;
			    case "false": t.kind = 11; break;
			    case "final": t.kind = 12; break;
			    case "float": t.kind = 13; break;
			    case "int": t.kind = 14; break;
			    case "long": t.kind = 15; break;
			    case "new": t.kind = 16; break;
			    case "null": t.kind = 17; break;
			    case "short": t.kind = 18; break;
			    case "static": t.kind = 19; break;
			    case "super": t.kind = 20; break;
			    case "this": t.kind = 21; break;
			    case "true": t.kind = 22; break;
			    case "void": t.kind = 23; break;
			    case "package": t.kind = 39; break;
			    case "import": t.kind = 41; break;
			    case "public": t.kind = 43; break;
			    case "protected": t.kind = 44; break;
			    case "private": t.kind = 45; break;
			    case "abstract": t.kind = 46; break;
			    case "strictfp": t.kind = 47; break;
			    case "native": t.kind = 48; break;
			    case "synchronized": t.kind = 49; break;
			    case "transient": t.kind = 50; break;
			    case "volatile": t.kind = 51; break;
			    case "extends": t.kind = 53; break;
			    case "implements": t.kind = 54; break;
			    case "throws": t.kind = 55; break;
			    case "interface": t.kind = 56; break;
			    case "if": t.kind = 57; break;
			    case "else": t.kind = 58; break;
			    case "for": t.kind = 59; break;
			    case "while": t.kind = 60; break;
			    case "do": t.kind = 61; break;
			    case "try": t.kind = 62; break;
			    case "finally": t.kind = 63; break;
			    case "switch": t.kind = 64; break;
			    case "return": t.kind = 65; break;
			    case "throw": t.kind = 66; break;
			    case "break": t.kind = 67; break;
			    case "continue": t.kind = 68; break;
			    case "catch": t.kind = 69; break;
			    case "case": t.kind = 70; break;
			    case "default": t.kind = 71; break;
			    case "instanceof": t.kind = 73; break;
			    default: break;
			}
		}

		bool Comment0()
		{
			int level = 1, pos0 = pos, line0 = line, col0 = col;
			NextCh();
			if (ch == '/') {
				NextCh();
				for (; ; ) {
					if (ch == 10) {
						level--;
						if (level == 0) {
							oldEols = line - line0; NextCh(); return true;
						}
						NextCh();
					} else if (ch == Buffer.EOF) return false;
					else NextCh();
				}
			} else {
				buffer.Pos = pos0; NextCh(); line = line0; col = col0;
			}
			return false;
		}

		bool Comment1()
		{
			int level = 1, pos0 = pos, line0 = line, col0 = col;
			NextCh();
			if (ch == '*') {
				NextCh();
				for (; ; ) {
					if (ch == '*') {
						NextCh();
						if (ch == '/') {
							level--;
							if (level == 0) {
								oldEols = line - line0; NextCh(); return true;
							}
							NextCh();
						}
					} else if (ch == Buffer.EOF) return false;
					else NextCh();
				}
			} else {
				buffer.Pos = pos0; NextCh(); line = line0; col = col0;
			}
			return false;
		}

		void Init()
		{
			pos = -1; line = 1; col = 0;
			oldEols = 0;
			NextCh();
			if (ch == 0xEF) { // check optional byte order mark for UTF-8
				NextCh(); int ch1 = ch;
				NextCh(); int ch2 = ch;
				if (ch1 != 0xBB || ch2 != 0xBF) {
					throw new FatalError(String.Format("illegal byte order mark: EF {0,2:X} {1,2:X}", ch1, ch2));
				}
				buffer = new UTF8Buffer(buffer); col = 0;
				NextCh();
			}
			start = new Dictionary<int, int>(128);
			for (int i = 36; i <= 36; ++i) start[i] = 1;
			for (int i = 65; i <= 90; ++i) start[i] = 1;
			for (int i = 95; i <= 95; ++i) start[i] = 1;
			for (int i = 97; i <= 122; ++i) start[i] = 1;
			for (int i = 48; i <= 48; ++i) start[i] = 46;
			for (int i = 49; i <= 57; ++i) start[i] = 47;
			start[46] = 48;
			start[39] = 18;
			start[34] = 27;
			start[58] = 35;
			start[44] = 36;
			start[45] = 73;
			start[43] = 74;
			start[123] = 39;
			start[91] = 40;
			start[40] = 41;
			start[33] = 75;
			start[125] = 42;
			start[93] = 43;
			start[41] = 44;
			start[126] = 45;
			start[59] = 54;
			start[42] = 76;
			start[61] = 77;
			start[63] = 55;
			start[47] = 78;
			start[38] = 79;
			start[124] = 80;
			start[94] = 81;
			start[37] = 82;
			start[60] = 83;
			start[62] = 84;
			start[Buffer.EOF] = -1;

			pt = tokens = new Token(); // first token is a dummy
		}

		void NextCh()
		{
			if (oldEols > 0) {
				ch = EOL; oldEols--;
			} else {
				pos = buffer.Pos;
				ch = buffer.Read(); col++;
				// replace isolated '\r' by '\n' in order to make
				// eol handling uniform across Windows, Unix and Mac
				if (ch == '\r' && buffer.Peek() != '\n') ch = EOL;
				if (ch == EOL) {
					line++; col = 0;
				}
			}
		}

		Token NextToken()
		{
			while (ch == ' ' ||
			       ch >= 9 && ch <= 10 || ch == 13
			       ) NextCh();
			if (ch == '/' && Comment0() || ch == '/' && Comment1()) return NextToken();
			t = new Token();
			t.pos = pos; t.col = col; t.line = line;
			int state;
			try { state = start[ch]; }
			catch (KeyNotFoundException) {
				state = 0;
			}
			tlen = 0; AddCh();

			switch (state) {
			    case -1: { t.kind = eofSym; break; } // NextCh already done
			    case 0: { t.kind = noSym; break; } // NextCh already done
			    case 1:
				    if (ch == '$' || ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch == '_' || ch >= 'a' && ch <= 'z') {
					    AddCh(); goto case 1;
				    } else {t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t; }
			    case 2:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 3;
				    } else {t.kind = noSym; break; }
			    case 3:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 3;
				    } else if (ch == 'L' || ch == 'l')                                                                                                           {
					    AddCh(); goto case 4;
				    } else {t.kind = 2; break; }
			    case 4:
			    { t.kind = 2; break; }
			    case 5:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 5;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                       {
					    AddCh(); goto case 17;
				    } else if (ch == 'E' || ch == 'e')                                                                                                                                             {
					    AddCh(); goto case 6;
				    } else {t.kind = 3; break; }
			    case 6:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 8;
				    } else if (ch == '+' || ch == '-')                                                       {
					    AddCh(); goto case 7;
				    } else {t.kind = noSym; break; }
			    case 7:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 8;
				    } else {t.kind = noSym; break; }
			    case 8:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 8;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                       {
					    AddCh(); goto case 17;
				    } else {t.kind = 3; break; }
			    case 9:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 9;
				    } else if (ch == '.')                                                       {
					    AddCh(); goto case 10;
				    } else if (ch == 'E' || ch == 'e')                                                                                                      {
					    AddCh(); goto case 14;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                                                                                                                                  {
					    AddCh(); goto case 17;
				    } else {t.kind = noSym; break; }
			    case 10:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 10;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                        {
					    AddCh(); goto case 17;
				    } else if (ch == 'E' || ch == 'e')                                                                                                                                              {
					    AddCh(); goto case 11;
				    } else {t.kind = 3; break; }
			    case 11:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 13;
				    } else if (ch == '+' || ch == '-')                                                        {
					    AddCh(); goto case 12;
				    } else {t.kind = noSym; break; }
			    case 12:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 13;
				    } else {t.kind = noSym; break; }
			    case 13:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 13;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                        {
					    AddCh(); goto case 17;
				    } else {t.kind = 3; break; }
			    case 14:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 16;
				    } else if (ch == '+' || ch == '-')                                                        {
					    AddCh(); goto case 15;
				    } else {t.kind = noSym; break; }
			    case 15:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 16;
				    } else {t.kind = noSym; break; }
			    case 16:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 16;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                        {
					    AddCh(); goto case 17;
				    } else {t.kind = 3; break; }
			    case 17:
			    { t.kind = 3; break; }
			    case 18:
				    if (ch <= 9 || ch >= 11 && ch <= 12 || ch >= 14 && ch <= '&' || ch >= '(' && ch <= '[' || ch >= ']' && ch <= 65535) {
					    AddCh(); goto case 19;
				    } else if (ch == 92)                                                                                                                                                {
					    AddCh(); goto case 20;
				    } else {t.kind = noSym; break; }
			    case 19:
				    if (ch == 39) {
					    AddCh(); goto case 26;
				    } else {t.kind = noSym; break; }
			    case 20:
				    if (ch >= '0' && ch <= '3') {
					    AddCh(); goto case 49;
				    } else if (ch >= '4' && ch <= '7')                                                        {
					    AddCh(); goto case 25;
				    } else if (ch == '"' || ch == 39 || ch == 92 || ch == 'b' || ch == 'f' || ch == 'n' || ch == 'r' || ch == 't')                                                                                                                    {
					    AddCh(); goto case 19;
				    } else if (ch == 'u')                                                                                                                                                                                                                                                            {
					    AddCh(); goto case 21;
				    } else {t.kind = noSym; break; }
			    case 21:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 22;
				    } else if (ch == 'u')                                                                                                            {
					    AddCh(); goto case 21;
				    } else {t.kind = noSym; break; }
			    case 22:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 23;
				    } else {t.kind = noSym; break; }
			    case 23:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 24;
				    } else {t.kind = noSym; break; }
			    case 24:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 19;
				    } else {t.kind = noSym; break; }
			    case 25:
				    if (ch >= '0' && ch <= '7') {
					    AddCh(); goto case 19;
				    } else if (ch == 39)                                                        {
					    AddCh(); goto case 26;
				    } else {t.kind = noSym; break; }
			    case 26:
			    { t.kind = 4; break; }
			    case 27:
				    if (ch <= 9 || ch >= 11 && ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) {
					    AddCh(); goto case 27;
				    } else if (ch == '"')                                                                                                                                                {
					    AddCh(); goto case 34;
				    } else if (ch == 92)                                                                                                                                                                                               {
					    AddCh(); goto case 28;
				    } else {t.kind = noSym; break; }
			    case 28:
				    if (ch >= '0' && ch <= '3') {
					    AddCh(); goto case 51;
				    } else if (ch >= '4' && ch <= '7')                                                        {
					    AddCh(); goto case 33;
				    } else if (ch == '"' || ch == 39 || ch == 92 || ch == 'b' || ch == 'f' || ch == 'n' || ch == 'r' || ch == 't')                                                                                                                    {
					    AddCh(); goto case 27;
				    } else if (ch == 'u')                                                                                                                                                                                                                                                            {
					    AddCh(); goto case 29;
				    } else {t.kind = noSym; break; }
			    case 29:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 30;
				    } else if (ch == 'u')                                                                                                            {
					    AddCh(); goto case 29;
				    } else {t.kind = noSym; break; }
			    case 30:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 31;
				    } else {t.kind = noSym; break; }
			    case 31:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 32;
				    } else {t.kind = noSym; break; }
			    case 32:
				    if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {
					    AddCh(); goto case 27;
				    } else {t.kind = noSym; break; }
			    case 33:
				    if (ch <= 9 || ch >= 11 && ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) {
					    AddCh(); goto case 27;
				    } else if (ch == '"')                                                                                                                                                {
					    AddCh(); goto case 34;
				    } else if (ch == 92)                                                                                                                                                                                               {
					    AddCh(); goto case 28;
				    } else {t.kind = noSym; break; }
			    case 34:
			    { t.kind = 5; break; }
			    case 35:
			    { t.kind = 24; break; }
			    case 36:
			    { t.kind = 25; break; }
			    case 37:
			    { t.kind = 26; break; }
			    case 38:
			    { t.kind = 28; break; }
			    case 39:
			    { t.kind = 29; break; }
			    case 40:
			    { t.kind = 30; break; }
			    case 41:
			    { t.kind = 31; break; }
			    case 42:
			    { t.kind = 35; break; }
			    case 43:
			    { t.kind = 36; break; }
			    case 44:
			    { t.kind = 37; break; }
			    case 45:
			    { t.kind = 38; break; }
			    case 46:
				    if (ch >= '0' && ch <= '7') {
					    AddCh(); goto case 53;
				    } else if (ch >= '8' && ch <= '9')                                                        {
					    AddCh(); goto case 9;
				    } else if (ch == 'L' || ch == 'l')                                                                                                                   {
					    AddCh(); goto case 4;
				    } else if (ch == 'X' || ch == 'x')                                                                                                                                                                              {
					    AddCh(); goto case 2;
				    } else if (ch == '.')                                                                                                                                                                                                                                         {
					    AddCh(); goto case 10;
				    } else if (ch == 'E' || ch == 'e')                                                                                                                                                                                                                                                                                        {
					    AddCh(); goto case 14;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                                                                                                                                                                                                                                                                                                                    {
					    AddCh(); goto case 17;
				    } else {t.kind = 2; break; }
			    case 47:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 47;
				    } else if (ch == 'L' || ch == 'l')                                                        {
					    AddCh(); goto case 4;
				    } else if (ch == '.')                                                                                                                   {
					    AddCh(); goto case 10;
				    } else if (ch == 'E' || ch == 'e')                                                                                                                                                                  {
					    AddCh(); goto case 14;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                                                                                                                                                                                              {
					    AddCh(); goto case 17;
				    } else {t.kind = 2; break; }
			    case 48:
				    if (ch >= '0' && ch <= '9') {
					    AddCh(); goto case 5;
				    } else {t.kind = 27; break; }
			    case 49:
				    if (ch >= '0' && ch <= '7') {
					    AddCh(); goto case 50;
				    } else if (ch == 39)                                                        {
					    AddCh(); goto case 26;
				    } else {t.kind = noSym; break; }
			    case 50:
				    if (ch >= '0' && ch <= '7') {
					    AddCh(); goto case 19;
				    } else if (ch == 39)                                                        {
					    AddCh(); goto case 26;
				    } else {t.kind = noSym; break; }
			    case 51:
				    if (ch <= 9 || ch >= 11 && ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '/' || ch >= '8' && ch <= '[' || ch >= ']' && ch <= 65535) {
					    AddCh(); goto case 27;
				    } else if (ch >= '0' && ch <= '7')                                                                                                                                                                          {
					    AddCh(); goto case 52;
				    } else if (ch == '"')                                                                                                                                                                                                                                      {
					    AddCh(); goto case 34;
				    } else if (ch == 92)                                                                                                                                                                                                                                                                                     {
					    AddCh(); goto case 28;
				    } else {t.kind = noSym; break; }
			    case 52:
				    if (ch <= 9 || ch >= 11 && ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) {
					    AddCh(); goto case 27;
				    } else if (ch == '"')                                                                                                                                                {
					    AddCh(); goto case 34;
				    } else if (ch == 92)                                                                                                                                                                                               {
					    AddCh(); goto case 28;
				    } else {t.kind = noSym; break; }
			    case 53:
				    if (ch >= '0' && ch <= '7') {
					    AddCh(); goto case 53;
				    } else if (ch >= '8' && ch <= '9')                                                        {
					    AddCh(); goto case 9;
				    } else if (ch == 'L' || ch == 'l')                                                                                                                   {
					    AddCh(); goto case 4;
				    } else if (ch == '.')                                                                                                                                                                              {
					    AddCh(); goto case 10;
				    } else if (ch == 'E' || ch == 'e')                                                                                                                                                                                                                             {
					    AddCh(); goto case 14;
				    } else if (ch == 'D' || ch == 'F' || ch == 'd' || ch == 'f')                                                                                                                                                                                                                                                                                         {
					    AddCh(); goto case 17;
				    } else {t.kind = 2; break; }
			    case 54:
			    { t.kind = 40; break; }
			    case 55:
			    { t.kind = 72; break; }
			    case 56:
			    { t.kind = 74; break; }
			    case 57:
			    { t.kind = 75; break; }
			    case 58:
			    { t.kind = 76; break; }
			    case 59:
			    { t.kind = 77; break; }
			    case 60:
			    { t.kind = 78; break; }
			    case 61:
			    { t.kind = 79; break; }
			    case 62:
			    { t.kind = 80; break; }
			    case 63:
			    { t.kind = 81; break; }
			    case 64:
			    { t.kind = 82; break; }
			    case 65:
			    { t.kind = 83; break; }
			    case 66:
			    { t.kind = 84; break; }
			    case 67:
			    { t.kind = 85; break; }
			    case 68:
			    { t.kind = 86; break; }
			    case 69:
			    { t.kind = 90; break; }
			    case 70:
			    { t.kind = 91; break; }
			    case 71:
			    { t.kind = 94; break; }
			    case 72:
			    { t.kind = 95; break; }
			    case 73:
				    if (ch == '-') {
					    AddCh(); goto case 37;
				    } else if (ch == '=')                                           {
					    AddCh(); goto case 57;
				    } else {t.kind = 32; break; }
			    case 74:
				    if (ch == '+') {
					    AddCh(); goto case 38;
				    } else if (ch == '=')                                           {
					    AddCh(); goto case 56;
				    } else {t.kind = 34; break; }
			    case 75:
				    if (ch == '=') {
					    AddCh(); goto case 70;
				    } else {t.kind = 33; break; }
			    case 76:
				    if (ch == '=') {
					    AddCh(); goto case 58;
				    } else {t.kind = 42; break; }
			    case 77:
				    if (ch == '=') {
					    AddCh(); goto case 69;
				    } else {t.kind = 52; break; }
			    case 78:
				    if (ch == '=') {
					    AddCh(); goto case 59;
				    } else {t.kind = 99; break; }
			    case 79:
				    if (ch == '=') {
					    AddCh(); goto case 60;
				    } else if (ch == '&')                                           {
					    AddCh(); goto case 68;
				    } else {t.kind = 89; break; }
			    case 80:
				    if (ch == '=') {
					    AddCh(); goto case 61;
				    } else if (ch == '|')                                           {
					    AddCh(); goto case 67;
				    } else {t.kind = 87; break; }
			    case 81:
				    if (ch == '=') {
					    AddCh(); goto case 62;
				    } else {t.kind = 88; break; }
			    case 82:
				    if (ch == '=') {
					    AddCh(); goto case 63;
				    } else {t.kind = 100; break; }
			    case 83:
				    if (ch == '<') {
					    AddCh(); goto case 85;
				    } else if (ch == '=')                                           {
					    AddCh(); goto case 71;
				    } else {t.kind = 92; break; }
			    case 84:
				    if (ch == '>') {
					    AddCh(); goto case 86;
				    } else if (ch == '=')                                           {
					    AddCh(); goto case 72;
				    } else {t.kind = 93; break; }
			    case 85:
				    if (ch == '=') {
					    AddCh(); goto case 64;
				    } else {t.kind = 96; break; }
			    case 86:
				    if (ch == '=') {
					    AddCh(); goto case 65;
				    } else if (ch == '>')                                           {
					    AddCh(); goto case 87;
				    } else {t.kind = 97; break; }
			    case 87:
				    if (ch == '=') {
					    AddCh(); goto case 66;
				    } else {t.kind = 98; break; }

			}
			t.val = new String(tval, 0, tlen);
			return t;
		}

		#endregion Methods
	}

	public class Token
	{
		#region Fields

		public int col; // token column (starting at 1)
		public int kind; // token kind
		public int line; // token line (starting at 1)
		public Token next; // ML 2005-03-11 Tokens are kept in linked list
		public int pos; // token position in the source text (starting at 0)
		public string val; // token value

		#endregion Fields
	}

	//-----------------------------------------------------------------------------------
	// UTF8Buffer
	//-----------------------------------------------------------------------------------
	public class UTF8Buffer : Buffer
	{
		#region Constructors

		public UTF8Buffer(Buffer b)
			: base(b)
		{
		}

		#endregion Constructors

		#region Methods

		public override int Read()
		{
			int ch;
			do {
				ch = base.Read();
				// until we find a uft8 start (0xxxxxxx or 11xxxxxx)
			} while ((ch >= 128) && ((ch & 0xC0) != 0xC0) && (ch != EOF));
			if (ch < 128 || ch == EOF) {
				// nothing to do, first 127 chars are the same in ascii and utf8
				// 0xxxxxxx or end of file character
			} else if ((ch & 0xF0) == 0xF0)   {
				// 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
				int c1 = ch & 0x07; ch = base.Read();
				int c2 = ch & 0x3F; ch = base.Read();
				int c3 = ch & 0x3F; ch = base.Read();
				int c4 = ch & 0x3F;
				ch = (((((c1 << 6) | c2) << 6) | c3) << 6) | c4;
			} else if ((ch & 0xE0) == 0xE0)   {
				// 1110xxxx 10xxxxxx 10xxxxxx
				int c1 = ch & 0x0F; ch = base.Read();
				int c2 = ch & 0x3F; ch = base.Read();
				int c3 = ch & 0x3F;
				ch = (((c1 << 6) | c2) << 6) | c3;
			} else if ((ch & 0xC0) == 0xC0)   {
				// 110xxxxx 10xxxxxx
				int c1 = ch & 0x1F; ch = base.Read();
				int c2 = ch & 0x3F;
				ch = (c1 << 6) | c2;
			}
			return ch;
		}

		#endregion Methods
	}
}